# coding=utf-8

#based on the algorithm described here: http://snowball.tartarus.org/algorithms/french/stemmer.html

import unittest, re

# API
def stem(word):
    return porter(word)


def isVowel(character):
    return character in [u'a',u'e',u'i',u'o',u'u',u'y',u'â',u'à',u'ë',u'é',u'ê',u'è',u'ï',u'î',u'ô',u'û',u'ù']

def getVowels():
    return [u'a',u'e',u'i',u'o',u'u',u'y',u'â',u'à',u'ë',u'é',u'ê',u'è',u'ï',u'î',u'ô',u'û',u'ù']
    
def getNotVowelsReg():
    return u"[^'a','e','i','o','u','y','â','à','ë','é','ê','è','ï','î','ô','û','ù']"

def isWithin(span, containingSpan):
    if containingSpan is None or span is None: return False
    return (span[0] >= containingSpan[0]) and (span[1] <= containingSpan[1])
    
def preprocess(word):
    #initial word preprpocessing
    #* go lower case
    word = word.lower()
    #word = unicode(word,"utf-8") 
    
    #    PUT TO UPPERCASE:
    #*   u or i preceded and followed by a vowel, 
    #*   y preceded or followed by a vowel. 
    #*   u after q is also put into upper case
 
    uWithVowels = re.compile(u"%s(u)%s" % (getVowels(),getVowels()),re.VERBOSE)
    iWithVowels = re.compile(u"%s(i)%s" % (getVowels(),getVowels()),re.VERBOSE)
    yWithVowelBefore = re.compile(u"%s(y)" % getVowels(),re.VERBOSE)
    yWithVowelAfter = re.compile(u"(y)%s" % getVowels(),re.VERBOSE)
    uAfterQ = re.compile(u"q(u)",re.VERBOSE)
    
    def searchNReplace(word,rexp,replaceChar):
        while True:
            m = rexp.search(word)
            if m:
                word = word[0:m.start(1)] + replaceChar + word[m.start(1)+1:]
            else: break
        return word
    
    word = searchNReplace(word,uWithVowels,"U")
    word = searchNReplace(word,iWithVowels,"I")
    word = searchNReplace(word,yWithVowelBefore,"Y")
    word = searchNReplace(word,yWithVowelAfter,"Y")
    word = searchNReplace(word,uAfterQ,"U")
    
    word = word.strip()
    
    return word

def getSpanAsString(span,originalString):
    if span is None: return ""
    return originalString[span[0]:span[1]+1] 
    
def getRv(word):
    ## RV is returned as a tuple (startIndex, endIndex)

    #if the word begins with two vowels, 
    #    RV is the region after the third letter, 
    #else 
    #    the region after the first vowel not at the beginning of the word, 
    #    or the end of the word if these positions cannot be found. 
    #    (Exceptionally, par, col or tap, at the begining of a word is also taken to define RV as 
    #     the region to their right.) 

    #the word starts with 2 vowels
    wordStartsW2Vowels = re.compile(u"^%s{2}" % getVowels(),re.VERBOSE)
    if wordStartsW2Vowels.match(word): 
        if len(word) > 3: return (3,len(word))    

    #EXCEPTIONS: "par", "col" or "tap" at the begining of a word is also taken to define RV as the region to their right
    isException = re.compile("^(par|col|tap)(.*)",re.VERBOSE)        
    m = isException.match(word)
    if m is not None:
        groupSpan = m.span(2)
        return (groupSpan[0], groupSpan[1]) 
    
    #take the region after the first vowel not at the beginning of the word
    def sectionAfter2ndVowel(t):
        regionAfterVowel = re.compile(unicode(u"(%s)+%s{1}(.*)" % (getNotVowelsReg(),getVowels())),re.VERBOSE)
        m = regionAfterVowel.match(t) 
        if m is not None:
            groupSpan = m.span(2)
            return (groupSpan[0], groupSpan[1])
        return None
    
    if isVowel(word[0]): 
        #the word starts with a vowel, find the second one and use the region after it
        #use temporary chunked word without starting vowel
        if len(word) > 1:
            t = word[1:len(word)]
            span = sectionAfter2ndVowel(t)
            if span is not None:
                return (span[0]+1,span[1]+1) #account for previous cutting
    else:
        return sectionAfter2ndVowel(word)
    
    return None

def getR1(word):
    #R1 is the region after the first non-vowel following a vowel, or the end of the word if there is no such non-vowel
    r1Reg = re.compile("%s*%s+%s{1}(\S*)" % (getNotVowelsReg(),getVowels(),getNotVowelsReg()),re.VERBOSE)
    m = r1Reg.match(word)
    if m is not None: return (m.span(1)[0],m.span(1)[1])
    return None

def getR2(word):
    #R2 is the region after the first non-vowel following a vowel in R1, or the end of the word if there is no such non-vowel.
    r1 = getR1(word) 
    if r1 is None: return None
    r2Reg = re.compile("%s*%s+%s{1}(\S*)" % (getNotVowelsReg(),getVowels(),getNotVowelsReg()),re.VERBOSE)
    m = r2Reg.match(word,r1[0])
    if m is not None: return (m.span(1)[0],m.span(1)[1]) 

#this is bad, move to a class
isStep2Needed = False
#Do step 2a if either no ending was removed by step 1, or if one of endings amment, emment, ment, ments was found. 
isStep2ANeeded = False
#Do step 2b if step 2a was done, but failed to remove a suffix. 
isStep2BNeeded = False


    
def step1(word,rv,r1,r2):
    #Standard suffix removal
    #Search for the longest among the following suffixes, and perform the action indicated.
    
    #this is bad, move to a class
    global isStep2Needed
    global isStep2ANeeded
    
    #Do step 2a if either no ending was removed by step 1, 
    #or if one of endings amment, emment, ment, ments was found. 
    
    # ance   iqUe   isme   able   iste   eux   ances   iqUes   ismes   ables   istes
        # delete if in R2
    suffixes = ["istes","ables","ismes","iqUes","ances","eux","iste","able","isme","iqUe","ance"]
    
    span = None
    for suffix in suffixes:
        if word.endswith(suffix): 
            span = (word.rfind(suffix),len(word))
            break
    
    if span is not None:
        if isWithin(span,r2):
            word = word[0:span[0]]
            return word
        
    #atrice   ateur   ation   atrices   ateurs   ations
        #delete if in R2 
        #if preceded by ic, delete if in R2, else replace by iqU 
    suffixes = ["ations", "ateurs", "atrices", "ation", "ateur", "atrice"]
    for suffix in suffixes:
        if word.endswith(suffix): 
            span = (word.rfind(suffix),len(word))
            if word.endswith("ic"+suffix):
                icSpan = (span[0]-2,span[0])
                if not isWithin(icSpan,r2):
                    word = word[0:span[0]] 
                    return word[0:len(word)-2]+"iqU"
                else: span = (span[0] - 2,span[1])    
            if isWithin(span,r2):
                return word[:-(span[1]-span[0])] 
        
    def replaceIfInR2(word,r2,suffixes,replacement):
        for suffix in suffixes:
            if word.endswith(suffix):
                span = (word.rfind(suffix),len(word))
                if isWithin(span,r2): return word.replace(suffix,replacement)
        return None

    #logie   logies
    #replace with log if in R2
    t = replaceIfInR2(word,r2,["logies","logie"],"log")
    if t is not None: return t
    
    #usion   ution   usions   utions
        #replace with u if in R2
    t = replaceIfInR2(word,r2,["utions","usions","ution","usion"],"u")
    if t is not None: return t
    
    #ence   ences
        #replace with ent if in R2 
    t = replaceIfInR2(word,r2,["ences","ence"],"ent")
    if t is not None: return t
    
    #issement   issements
    #delete if in R1 and preceded by a non-vowel 
    suffixes = ["issements","issement"]
    for suffix in suffixes:
        if word.endswith(suffix):
            span = (len(word)-len(suffix),len(word))
            if isWithin(span,r1) and not isVowel(word[len(word)-len(suffix)-1]):
                return word[:-len(suffix)]
            else: return word
    #ement   ements
        #delete if in RV 
        #if preceded by iv, delete if in R2 (and if further preceded by at, delete if in R2), otherwise, 
        #if preceded by eus, delete if in R2, else replace by eux if in R1, otherwise, 
        #if preceded by abl or iqU, delete if in R2, otherwise, 
        #if preceded by i�r or I�r, replace by i if in RV 
    
    suffixes = ["ements","ement"]
    span2Delete = None
    for suffix in suffixes:
        if word.endswith(suffix): 
            span = (word.rfind(suffix),len(word))
            if isWithin(span,rv): 
                span2Delete = span
            
            #iv
            if word.endswith("iv" + suffix):
                ivSpan = (span[0]-2,span[0])
                if isWithin(ivSpan,r2):
                    if span2Delete is None: span2Delete = ivSpan
                    else: span2Delete = (ivSpan[0],len(word))
                
                if word.endswith("ativ" + suffix):
                    atSpan = (span[0]-4,span[0]-2)
                    if isWithin(atSpan,r2):
                        span2Delete = (span[0]-4,len(word))
                
            #eus
            if word.endswith("eus" + suffix):
                eusSpan = (span[0]-3,span[0])
                if isWithin(eusSpan,r2):
                    if span2Delete is None: span2Delete = eusSpan
                    else: span2Delete = (eusSpan[0],len(word))
                else:
                    if isWithin(eusSpan,r1):
                        #replace eus by eux
                        word = word.replace("eus","eux")
            #abl or iqU    
            if word.endswith("abl"+suffix) or word.endswith("iqU"+suffix):
                tSpan = (span[0]-3,span[0])
                if isWithin(tSpan,r2):
                    if span2Delete is None: span2Delete = tSpan
                    else: span2Delete = (tSpan[0],len(word))
            
            #ièr or Ièr
            if word.endswith(u"ièr"+suffix) or word.endswith(u"Ièr"+suffix):
                tSpan = (span[0]-3,span[0])
                if isWithin(tSpan,rv):
                    return word[:len(word)-len(suffix)-3] + "i"
                    
            break
    
    if span2Delete is not None:
        return word[0:span2Delete[0]]
    
    #ité   ités
        #delete if in R2 
        #if preceded by abil, delete if in R2, else replace by abl, otherwise, 
        #if preceded by ic, delete if in R2, else replace by iqU, otherwise, 
        #if preceded by iv, delete if in R2

    suffixes = [u"ités", u"ité"]
    span2Delete = None
    for suffix in suffixes:
        if word.endswith(suffix):
            span = (word.rfind(suffix),len(word))
            if isWithin(span,r2):
                span2Delete = span
                
            #abil
            if word.endswith("abil"+suffix):
                abilSpan = (span[0]-4,span[0])
                if isWithin(abilSpan,r2):
                    span2Delete = (abilSpan[0],len(word))
                else:
                    word = word.replace(word[abilSpan[0]:abilSpan[1]],"abl")
                    span2Delete = [word.rfind(suffix),len(word)]
                break;    
            #ic
            if word.endswith("ic"+suffix):
                icSpan = (span[0]-2,span[0])
                if isWithin(icSpan,r2):
                    span2Delete = (icSpan[0],len(word))
                else:
                    word = word.replace(word[icSpan[0]:icSpan[1]],"iqU")
                    span2Delete = [word.rfind(suffix),len(word)]
                break;
                
            #iv
            if word.endswith("iv"+suffix):
                ivSpan = (span[0]-2,span[0])
                if isWithin(ivSpan,r2):
                    span2Delete = (ivSpan[0],len(word))
                break;
                    
            break;
            
    if span2Delete is not None:
        return word[0:span2Delete[0]]
    
    #if   ive   ifs   ives
        #delete if in R2 
        #if preceded by at, delete if in R2 (and if further preceded by ic, delete if in R2, else replace by iqU)
    suffixes = ["ives","ifs","ive","if"]
    span2Delete = None
    for suffix in suffixes:
        if word.endswith(suffix):
            span = (word.rfind(suffix),len(word))
            if isWithin(span,r2):
                span2Delete = span
            
            #at
            if word.endswith("at"+suffix):
                atSpan = (span[0]-2,span[0])
                if isWithin(atSpan,r2):
                    span2Delete = (atSpan[0],len(word))
                    
                    #ic
                    if word.endswith("icat"+suffix):
                        icSpan = (atSpan[0]-2,atSpan[0])
                        if isWithin(icSpan,r2):
                            span2Delete = (icSpan[0],len(word))
                        else:
                            word = word[0:icSpan[0]]+"iqU"+word[icSpan[1]:len(word)]
                            span2Delete = (span2Delete[0]+1,len(word))
            break;
    
    if span2Delete is not None:
        return word[0:span2Delete[0]]
    
    #eaux replace with eau 
    if word.endswith("eaux"):
        return word.replace("eaux","eau")
    
    #aux replace with al if in R1
    if word.endswith("aux"):
        span = (len(word)-3,len(word))
        if isWithin(span,r1):
            return word[:-3] + 'al'
    
    #euse   euses
    #delete if in R2, else replace by eux if in R1 
    suffixes = ["euses","euse"]
    for suffix in suffixes:
        if word.endswith(suffix):
            span = (len(word)-len(suffix),len(word))
            if isWithin(span,r2): return word[:-len(suffix)]
            else: 
                if isWithin(span,r1): return word[:-len(suffix)] + "eux"
    
    #amment
    #replace with ant if in RV 
    if word.endswith("amment"):
        isStep2ANeeded = True
        if isWithin((len(word)-6,len(word)),rv):
            return word[:-6] + "ant"
    
    #emment
    #replace with ent if in RV 
    if word.endswith("emment"):
        isStep2ANeeded = True
        if isWithin((len(word)-6,len(word)),rv):
            return word[:-6] + "ent"
    
    #ment   ments
    #delete if preceded by a vowel in RV
    suffixes = ["ments","ment"]
    for suffix in suffixes:
        if word.endswith(suffix):
            isStep2ANeeded = True
            if isVowel(word[len(word)-len(suffix)-1]):
                if isWithin((len(word)-len(suffix)-1,len(word)),rv):
                    return word[:-len(suffix)]
    
    #no ending has been removed if we reach this point, schedule 2nd step
    isStep2Needed = True
    
    #Do step 2a if either no ending was removed by step 1, or if one of endings amment, emment, ment, ments was found.
    isStep2ANeeded = True
    
    return word

def step2a(word,rv):
    #all tests are confined to the RV region. 
    
    global isStep2BNeeded
    
    # Verb suffixes beginning i
    #Search for the longest among the following suffixes and if found, delete if preceded by a non-vowel. 
    # îmes   ît   îtes   i   ie   ies   ir   ira   irai   iraIent   irais   irait   iras   irent   irez   iriez   irions   irons   iront   is   issaIent   issais   issait   issant   issante   issantes   issants   isse   issent   isses   issez   issiez   issions   issons   it
    #(Note that the non-vowel itself must also be in RV.)
    
    suffixes = ["issantes","issaIent","issants","iraIent","issante","issions","irions","issons","issais","issait","issant","issiez","issent","irons","iront","irais","irait","irent","isses","issez","iriez","irez","iras","irai","isse",u"îmes",u"îtes","ira","ies","ir","is","it","ie",u"ît","i"]
    
    for suffix in suffixes:
        if word.endswith(suffix): 
            #we offset the left position by 1 since the non-wovel test that follows assumes that the character
            #must be within the rv
            suffixLeft = len(word) - len(suffix)
            span = (suffixLeft-1,len(word)-1)
            if isWithin(span,rv):
                if suffixLeft > 0:
                    if not isVowel(word[suffixLeft-1]):
                        return word[:-len(suffix)]
    
    isStep2BNeeded = True
    return word
    
def step2b(word,rv,r2):
    #Search for the longest among the following suffixes (within RV), and perform the action indicated. 
    
    # é ée ées és èrent er era erai eraIent erais erait eras erez eriez erions erons eront ez iez
    #delete
    
    #TODO: !!! Iez added 
    suffixes = ["eraIent","erions","erais","erait",u"èrent","eriez","erons","eront","erai","eras","erez",u"ées","era","iez","Iez","er","ez",u"és",u"ée",u"é"]   
    for suffix in suffixes:
        if word.endswith(suffix):
            if isWithin((word.rfind(suffix),len(word)),rv):
                return word[:-len(suffix)]
    
    #âmes ât âtes a ai aIent ais ait ant ante antes ants as asse assent asses assiez assions
    #   = > delete
    # if preceded by e, delete
    #(Note that the e that may be deleted in this last step must also be in RV.)
    
    suffixes = ["assions","assiez","assent","aIent","asses","antes",u"âmes",u"âtes","ante","ants","asse","ais","ait","ant","as",u"ât","ai","a"]   
      
    for suffix in suffixes:
        if word.endswith(suffix):
            suffixLeft = word.rfind(suffix)
            span = (suffixLeft,len(word))
            if isWithin(span,rv):
                if suffixLeft > 0:
                    if word[suffixLeft -1] == "e":
                        if isWithin((suffixLeft -1,len(word)),rv):
                            return word[:-(len(suffix)+1)]
                return word[:-len(suffix)]
        
    # ions
    # delete if in R2 
    if word.endswith("ions"):
        if isWithin((word.rfind("ions"),len(word)),r2):
            return word[:-len("ions")]
    
    return word
    
def step3(word):
    #Replace final Y with i or final ç with c
    if word.endswith("Y"): return word[:-1] + "i"
    if word.endswith(u"ç"): return word[:-1] + "c"
    
    return word
    
def step4(word,rv,r2):
    #If the word ends s, not preceded by a, i, o, u, è or s, delete it.
    if word.endswith('s'):
        if word[len(word)-2] not in ('a','i','o','u',u'è','s'):
            word = word[:-1]
            
    #In the rest of step 4, all tests are confined to the RV region. 

    #Search for the longest among the following suffixes, and perform the action indicated. 

    #ion
    #delete if in R2 and preceded by s or t 
    if word.endswith('ion'):
        span = (word.rfind('ion'),len(word))
        if isWithin(span,rv) and isWithin(span,r2):
            if word[len(word)-4] in ['s','t']:
                if isWithin((word.rfind('ion')-1,len(word)),rv):
                    return word[:-3]
    
    #ier   ière   Ier   Ière
    #replace with i 
    suffixes = [u"Ière", u"ière", "ier", "Ier"]   
    for suffix in suffixes:
        if word.endswith(suffix):
            span = (word.rfind(suffix),len(word))
            if isWithin(span,rv):
                return word[:-len(suffix)] + "i"
    #e
    #delete 
    if word.endswith("e"):
        if isWithin((len(word)-1,len(word)),rv):
            return word[:-1]
    
    #ë
    #if preceded by gu, delete
    if word.endswith(u"guë"):
        if isWithin((len(word)-3,len(word)),rv):
            return word[:-1]
    
    return word

def step5(word):
    #Undouble
    #If the word ends enn, onn, ett, ell or eill, delete the last letter
    suffixes = ["enn", "onn", "ett", "ell","eill"]
    for suffix in suffixes:
        if word.endswith(suffix):
            return word[:-1]
    return word
    
def step6(word):
    #If the words ends é or è followed by at least one non-vowel, remove the accent from the e..
    r = re.compile(u".*(['é','è'])[^'a','e','i','o','u','y','â','à','ë','é','ê','è','ï','î','ô','û','ù']+$",re.VERBOSE)
    
    m = r.match(word)
    if m is not None:
        eIndex = m.span(1)[0]
        word = word[0:eIndex] + "e" + word[eIndex+1:len(word)]
    
    #Turn any remaining I, U and Y letters in the word back into lower case.
    return word.lower()
    
def porter(word):
    global isStep2Needed
    global isStep2ANeeded
    global isStep2BNeeded
    
    isStep2Needed = False
    isStep2ANeeded = False
    isStep2BNeeded = False
    
    word = preprocess(word)
    originalWord = word
    
    word = step1(word,getRv(word),getR1(word),getR2(word))
    
    if isStep2ANeeded:
        word = step2a(word,getRv(word))
    
    if isStep2BNeeded:
        word = step2b(word,getRv(word),getR2(word))
        
    if word != originalWord:
        word = step3(word)
    else: word = step4(word,getRv(word),getR2(word))
    
    word = step5(word)
    word = step6(word)
    
    return word
    
#tests
class TestPorter(unittest.TestCase):
    def setUp(self):
        pass

    def dumpStemmingSteps(self,word,processedWord,result,rv,r1,r2):
        print "-------------------------------------------------"
        print "stemming: %s" % word
        print "processed: %s" % processedWord
        print "result: %s" % result
        print "rv: %s => %s" % (rv,getSpanAsString(rv,word))
        print "r1: %s => %s" % (r1,getSpanAsString(r1,word))
        print "r2: %s => %s" % (r2,getSpanAsString(r2,word))
        print "-------------------------------------------------"

    def _loadTestingSet(self, fileName = "stems.txt"):
            #returns a dictionary originalForm => stemmedForm
        result = {}
        file = open(fileName, 'r')
        wordNStemReg = re.compile("^(\S*)\s+(.*)$",re.VERBOSE)
    
        while True:
            wordNStem = file.readline()
            if wordNStem == '': break
            m = wordNStemReg.match(wordNStem)
            if m is not None: result[m.group(1)] = m.group(2)
        
        return result
        
    def testVowels(self):
        for char in [u'a',u'e',u'i',u'o',u'u',u'y',u'â',u'à',u'ë',u'é',u'ê',u'è',u'ï',u'î',u'ô',u'û',u'ù']:
            self.assertTrue(isVowel(char))
        for char in [u'b',u'c',u'd',u'f',u'g',u'h',u'j',u'k',u'l',u'm',u'n',u'p',u'q',u'r',u's',u't',u'v',u'w',u'x',u'z']:
            self.assertTrue(isVowel(char) == False)

    def testPreprocess(self):
        #lowercase
        self.assertEqual(preprocess("BONJOUR"),"bonjour")
        
        #vowel fixing
        self.assertEqual(preprocess("jouer"),"joUer")
        self.assertEqual(preprocess("ennuie"),"ennuIe")
        self.assertEqual(preprocess("yeux"),"Yeux")
        self.assertEqual(preprocess("joyeux"),"joYeux")
        self.assertEqual(preprocess("quand"),"qUand")
        
    def __testGetRV(self):
        self.assertEqual(getSpanAsString(getRv("aimer"),"aimer"),"er")
        self.assertEqual(getSpanAsString(getRv("adorer"),"adorer"),"rer")
        self.assertEqual(getSpanAsString(getRv("voler"),"voler"),"ler")
        self.assertEqual(getSpanAsString(getRv("tapis"),"tapis"),"is")
    
    def __testGetR1(self):
        self.assertEqual(getSpanAsString(getR1("fameusement"),"fameusement"),"eusement")
    
    def __testGetR2(self):
        self.assertEqual(getSpanAsString(getR2("fameusement"),"fameusement"),"ement")
    
    def __testStep1(self):
        words2stems = {"abondance":"abond","acoustiques":"acoust","anachronisme":"anachron", "applicables" : "applic","audacieux" : "audaci", "application":"appliqu", "adjudication":   "adjud","adoratrices":"ador","accusateur":"accus","pr�dicateur":"pr�diqu","apparence":"apparent", "abattement":"abatt", "aboiements":"aboi","activement":"activ", "attentivement":"attent", "comparativement":"compar","amoureusement":  "amour","copieusement":"copieux","abominablement":"abomin","accablement":"accabl","despotiquement":"despot","famili�rement":"famili","absurdit�":     "absurd","affabilit�":"affabl","authenticit�":"authent","complicit�":"compliqu","activit�":"activ","afflictive":"afflict","alternative":"altern","communicative":"commun","vindicative":"vindiqu","agneaux":"agneau","animaux":"animal","affreuse":"affreux","affreuses":"affreux","ambitieuse":"ambiti","abrutissement":"abrut","apparemment":"apparent","assid�ment":"assid�"}
        #DOES NOT WORK: "abondamment":"abond", "brillamment":"brill"
        #DOES NOT WORK: "assid�ment":"assid�" = � is not considered as a vowel 
        
        #"astrologie":"astrolog"
        #"g�n�alogie":"g�n�alog"
        #"allusion":"allus"
        #"abaissement":"abaissement"
        #"enti�rement":"entier"
        
        for key, value in words2stems.items():
            word = preprocess(key)
            result = step1(word,getRv(word),getR1(word),getR2(word))
            
            if result.lower() != unicode(value,"utf-8"):
                self.dumpStemmingSteps(word,word,result,getRv(word),getR1(word),getR2(word))
                self.assertTrue(False)
            
    def __testStep2(self):
        
        #test step 2a
        
        words2stems = {"affadissantes":"affad","choisirai":"chois"}
        
        for key, value in words2stems.items():
            word = preprocess(key)
            result = step2a(word,getRv(word))
            
            if result.lower() != unicode(value,"utf-8"):
                self.dumpStemmingSteps(key,word,result,getRv(word),getR1(word),getR2(word))
                self.assertTrue(False)
    
        #test step 2b
        
        words2stems = {"rencontrassent":"rencontr","arriverions":"arriv","ablutions":"ablut"}
        
        for key, value in words2stems.items():
            word = preprocess(key)
            result = step2b(word,getRv(word),getR2(word))
            
            if result.lower() != unicode(value,"utf-8"):
                self.dumpStemmingSteps(key,word,result,getRv(word),getR1(word),getR2(word))
                self.assertTrue(False)
    
    def __testPorter(self):
        words2stems = {"rencontrassent":"rencontr","arriverions":"arriv","ablutions":"ablut","annon�ait":"annonc","aboiera":"aboi","accueillait":"accueil"}
        
        for key, value in words2stems.items():
            word = preprocess(key)
            result = porter(key)
            
            if result.lower() != unicode(value,"utf-8"):
                self.dumpStemmingSteps(key,word,result,getRv(word),getR1(word),getR2(word))
                self.assertTrue(False)
                
    def testPorterWithTestDataset(self):    
        data = self._loadTestingSet()
        
        counter = 0
        
        for key, value in data.items():
            try:
                result = porter(key)
            except Exception, x:
                import traceback
                print "problem stemming: %s :: %s" % (key,x) 
                traceback.print_exc()
                return
                
            if result.lower() != unicode(value,"utf-8"):
                result = porter(key)
                
                word = preprocess(key)
                self.dumpStemmingSteps(key,word,result,getRv(word),getR1(word),getR2(word))
                self.assertTrue(False)
            
            #print u"%s => %s" % (unicode(key, "unicode" ),unicode(result, "unicode" ))
                
            counter = counter + 1
    
        print "Tested with %s words" % counter
    
if __name__ == '__main__':
    unittest.main()

    
    
    
  
   
   
  
  
  
    